# -*- coding: utf-8 -*-
import scrapy

import re
import json
import logging
import datetime
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem
from FDC_spider.constants import HZ_NUM_DICT_1, HZ_NUM_DICT_2, HZ_NUM_DICT_3, HZ_EG_STATUS_DICT

logger = logging.getLogger(__name__)


class HuzhouSpider(scrapy.Spider):
    name = 'huzhou'
    allowed_domains = ['hu.tmsf.com']
    start_urls = ['http://hu.tmsf.com/newhouse/property_searchall.htm']
    cert_detail_temp_url = 'http://hu.tmsf.com/newhouse/NewPropertyHz_createPresellInfo.jspx?presellid={}&propertyid={}&sid={}&_={}'  # 许可证详情url
    bd_li_temp_url = 'http://hu.tmsf.com/newhouse/property_{}_{}_price.htm?isopen=&presellid={}&buildingid=&area=&allprice=&housestate=&housetype=&page=&roomid='  # 楼栋列表url

    def parse(self, response):
        page_str = response.xpath("//div[@class='pagenuber_info']/font[2]/text()").extract_first()
        page_li = re.findall(r'\d+', page_str)
        current_page = page_li[0]
        total_page = page_li[1]
        # 获取当前页楼盘分组
        eg_li = response.xpath("//div[@class='searchpageall']//ul/li")
        if len(eg_li):
            for eg in eg_li:
                item_eg = FdcEstateGuidItem()
                item_eg['estateName'] = eg.xpath(".//div[@class='build_txt']/h3/a/text()").extract_first()
                item_eg['estateUrl'] = eg.xpath(".//div[@class='build_txt']/h3/a/@href").extract_first()
                estateStatus_img = eg.xpath(".//div[@class='build_txt']/h3//@src").extract_first()
                item_eg['estateStatus'] = HZ_EG_STATUS_DICT.get(estateStatus_img, None)
                item_eg['propertyType'] = eg.xpath(".//div[@class='build_txt']/div[1]/p/text()").extract_first()
                item_eg['propertyType'] = item_eg['propertyType'].strip() if item_eg['propertyType'] else None
                item_eg['estateAddress'] = eg.xpath(".//div[@class='build_txt']/div[2]/p/text()").extract_first()
                item_eg['estateAddress'] = item_eg['estateAddress'].strip() if item_eg['estateAddress'] else None
                districtName_li = re.findall(r'[[](.*)[]]', item_eg['estateAddress']) if item_eg[
                    'estateAddress'] else []
                item_eg['districtName'] = districtName_li[0] if len(districtName_li) else None
                item_eg['developerName'] = eg.xpath(".//div[@class='build_txt']/div[3]/p/text()").extract_first()
                item_eg['developerName'] = item_eg['developerName'].strip() if item_eg['developerName'] else None
                item_eg['salesTel'] = eg.xpath(".//div[@class='build_txt']/div[4]/font[2]/text()").extract_first()
                item_eg['salesTel'] = item_eg['salesTel'].strip() if item_eg['salesTel'] else None
                avg_p_li = eg.xpath(".//div[@class='build_txt']/div[5]/p[1]/span/@class").extract()
                avg_p_str = ''.join([HZ_NUM_DICT_1.get(i, '') for i in avg_p_li])
                item_eg['avgPrice'] = float(avg_p_str) if avg_p_str and float(avg_p_str) else None
                cum_avg_p_li = eg.xpath(".//div[@class='build_txt']/div[5]/p[2]/span/@class").extract()
                cum_avg_p_str = ''.join([HZ_NUM_DICT_2.get(i, '') for i in cum_avg_p_li])
                item_eg['cumulativeAvgPrice'] = float(cum_avg_p_str) if cum_avg_p_str and float(cum_avg_p_str) else None
                item_eg['saleableRoomNo'] = eg.xpath(".//div[@class='build_pic']/div/p/a[1]/text()").extract_first()
                totalRoomNo_str = eg.xpath(".//div[@class='build_pic']/div/p/a[2]/text()").extract_first()
                totalRoomNo_li = re.findall(r'\d+', totalRoomNo_str)
                item_eg['totalRoomNo'] = totalRoomNo_li[0] if len(totalRoomNo_li) else 0
                yield response.follow(
                    item_eg['estateUrl'],
                    callback=self.parse_estate,
                    meta=dict(item_eg=deepcopy(item_eg)),
                    priority=2,
                )
        else:
            logger.warning('第{}页 楼盘列表获取为空'.format(current_page))

        # 翻页
        print('>>>>>>>>第{}/{}页<<<<<<<<'.format(current_page, total_page))
        if int(current_page) < int(total_page):
            # 构造POST请求参数
            data = dict(
                keytype=str(1),
                keyword='',
                sid=str(330500),
                districtid='',
                areaid='',
                dealprice='',
                propertystate='',
                propertytype='',
                ordertype='',
                priceorder='',
                openorder='',
                page=str(int(current_page) + 1),
                bbs=''
            )
            yield scrapy.FormRequest(
                response.request.url,
                formdata=data,
                priority=1,
            )

    def parse_estate(self, response):
        """
        获取楼盘详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['estateUrl'] = response.request.url
        item_eg['buildingForm'] = response.xpath("//strong[contains(text(),'建筑形式')]/../text()").extract_first()
        item_eg['floorAreaRatio'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'容 积 率')]/text()").extract_first()
        item_eg['greeningRate'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'绿 化 率')]/text()").extract_first()
        item_eg['decorateSituation'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'装修情况')]/text()").extract_first()
        item_eg['estateTotalArea'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'总建筑面积')]/text()").extract_first()
        item_eg['totalHouseholds'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'总户数')]/text()").extract_first()
        item_eg['propertyCompany'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'物业公司')]/text()").extract_first()
        item_eg['propertyManagementFee'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'物 业 费')]/text()").extract_first()
        item_eg['landUsageTerm'] = response.xpath(
            "//div[@class='buildshowin_contant ft14 colordg']/ul/li[contains(@title,'产权年限')]/text()").extract_first()
        # 获取楼盘详情页许可证分组
        tr_li = response.xpath("//div[@class='buildshow_contant colordg ft14']/table//tr[position()>1]")
        cert_date_dict = dict()
        if len(tr_li):
            for tr in tr_li:
                presale_permit = tr.xpath("./td[1]/a/text()").extract_first()
                cert_date = tr.xpath("./td[2]/text()").extract_first()
                if presale_permit:
                    cert_date_dict[presale_permit] = cert_date
        # 获取许可证列表url
        cert_li_url = response.xpath(
            "//h1[contains(text(),'楼盘申领预售证信息')]/preceding-sibling::span[1]/a/@href").extract_first()
        if cert_li_url:
            yield response.follow(
                cert_li_url,
                callback=self.parse_cert_li,
                meta=dict(item_eg=deepcopy(item_eg), cert_date_dict=deepcopy(cert_date_dict)),
                priority=3,
            )
        else:
            item_eg['preSalePermit'] = None
            item_eg['permitId'] = None
            item_eg['certDate'] = None
            item_eg['preSaleRoomNo'] = None
            item_eg['saleableRoomNo'] = None
            item_eg['saleableResidenceNum'] = None
            yield item_eg

    def parse_cert_li(self, response):
        """
        获取许可证列表、许可证下的楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        cert_date_dict = copy(response.meta['cert_date_dict'])
        # 获取许可证分组
        a_li = response.xpath("//div[@id='presell_dd']/div/a[position()>1]")
        if len(a_li):
            for a in a_li:
                item_eg['preSalePermit'] = a.xpath("./text()").extract_first()
                item_eg['certDate'] = cert_date_dict.get(item_eg['preSalePermit'], None) if item_eg[
                    'preSalePermit'] else None
                permitid_str = a.xpath("./@id").extract_first()
                permitid_li = re.findall(r'presell_(.*)', permitid_str)
                permitid = permitid_li[0] if len(permitid_li) else None
                item_eg['permitId'] = permitid
                p1, p2 = re.findall(r'property_(.*)_(.*)_', response.request.url)[0]
                p3 = int(round(datetime.datetime.now().timestamp() * 1000))
                if permitid:
                    # 获取许可证详情
                    yield scrapy.Request(
                        HuzhouSpider.cert_detail_temp_url.format(permitid, p2, p1, p3),
                        callback=self.parse_cert_detail,
                        meta=dict(item_eg=deepcopy(item_eg)),
                        priority=8,
                    )
                    # 获取楼栋列表
                    yield scrapy.Request(
                        HuzhouSpider.bd_li_temp_url.format(p1, p2, permitid),
                        callback=self.parse_bd_li,
                        meta=dict(item_eg=deepcopy(item_eg)),
                        priority=4,
                    )
        else:
            logger.warning(
                '{}  {}-{} 许可证列表获取为空'.format(response.request.url, item_eg['districtName'], item_eg['estateName']))

    def parse_cert_detail(self, response):
        """
        获取许可证详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        resp_dict = json.loads(response.body)
        pre_dict = resp_dict.get('pre', None)
        item_eg['preSaleRoomNo'] = pre_dict.get('num', None) if pre_dict else None
        item_eg['saleableRoomNo'] = pre_dict.get('avanum', None) if pre_dict else None
        item_eg['saleableResidenceNum'] = pre_dict.get('resideavanum', None) if pre_dict else None
        yield item_eg

    def parse_bd_li(self, response):
        """
        获取楼栋列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取楼栋分组
        a_li = response.xpath("//div[@id='building_dd']/div/a[position()>1]")
        if len(a_li):
            for a in a_li:
                item_bd = FdcBuildingItem()
                item_bd['districtName'] = item_eg['districtName']
                item_bd['estateName'] = item_eg['estateName']
                item_bd['preSalePermit'] = item_eg['preSalePermit']
                item_bd['permitId'] = item_eg['permitId']
                item_bd['blockName'] = a.xpath("./text()").extract_first()
                buildingId_str = a.xpath("./@id").extract_first()
                buildingId_li = re.findall(r'building_(.*)', buildingId_str)
                buildingId = buildingId_li[0] if len(buildingId_li) else None
                item_bd['buildingId'] = buildingId
                yield item_bd

                # 获取房号列表
                rm_li_temp_url = str(response.request.url).split('&buildingid')[
                                     0] + '&buildingid={}&area=&allprice=&housestate=&housetype=&page=&roomid='
                yield scrapy.Request(
                    rm_li_temp_url.format(buildingId),
                    callback=self.parse_rm_li,
                    meta=dict(item_bd=deepcopy(item_bd)),
                    priority=5,
                )
        else:
            logger.warning(
                '{}  {}-{}-{} 获取楼栋分组为空'.format(response.request.url, item_eg['districtName'], item_eg['estateName'],
                                               item_eg['preSalePermit']))

    def parse_rm_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        page_str = response.xpath("//div[@class='spagenext']/span/text()").extract_first()
        page_li = re.findall(r'\d+', page_str)
        current_page = page_li[0]
        total_page = page_li[1]
        # 获取房号分组
        tr_li = response.xpath("//div[@class='onbuildshow_contant colordg ft14']/table/tbody/tr")
        if len(tr_li):
            for tr in tr_li:
                item_rm = FdcRoomItem()
                item_rm['districtName'] = item_bd['districtName']
                item_rm['estateName'] = item_bd['estateName']
                item_rm['preSalePermit'] = item_bd['preSalePermit']
                item_rm['blockName'] = item_bd['blockName']
                item_rm['buildingId'] = item_bd['buildingId']
                item_rm['roomNo'] = tr.xpath("./td[2]//div/text()").extract_first()
                item_rm['roomArea'] = tr.xpath("./td[3]//div/text()").extract_first()
                item_rm['saleStatus'] = tr.xpath("./td[9]//a/text()").extract_first()
                item_rm['saleStatus'] = re.sub(r'\u3000', '', item_rm['saleStatus']) if item_rm['saleStatus'] else None
                roomArea_li = tr.xpath("./td[3]//div/span/@class").extract()
                roomArea_str = ''.join([HZ_NUM_DICT_3.get(i, '') for i in roomArea_li])
                item_rm['roomArea'] = float(roomArea_str) if roomArea_str and float(roomArea_str) else None
                innerArea_li = tr.xpath("./td[4]//div/span/@class").extract()
                innerArea_str = ''.join([HZ_NUM_DICT_3.get(i, '') for i in innerArea_li])
                item_rm['innerArea'] = float(innerArea_str) if innerArea_str and float(innerArea_str) else None
                occupancyRate_li = tr.xpath("./td[5]//div/span/@class").extract()
                occupancyRate_str = ''.join([HZ_NUM_DICT_3.get(i, '') for i in occupancyRate_li])
                item_rm['occupancyRate'] = occupancyRate_str + '%' if occupancyRate_str else None
                unitPrice_li = tr.xpath("./td[6]//div/span/@class").extract()
                unitPrice_str = ''.join([HZ_NUM_DICT_3.get(i, '') for i in unitPrice_li])
                item_rm['unitPrice'] = float(unitPrice_str) if unitPrice_str and float(unitPrice_str) else None
                roomTotalPrice_li = tr.xpath("./td[8]//div/span/@class").extract()
                roomTotalPrice_str = ''.join([HZ_NUM_DICT_3.get(i, '') for i in roomTotalPrice_li])
                item_rm['roomTotalPrice'] = float(roomTotalPrice_str) if roomTotalPrice_str and float(
                    roomTotalPrice_str) else None
                room_url = tr.xpath("./td[2]//a/@href").extract_first()
                yield response.follow(
                    room_url,
                    callback=self.parse_room_detail,
                    meta=dict(item_rm=deepcopy(item_rm)),
                    priority=7,
                )
        else:
            logger.warning('{}  {}-{}-{}-{} 第{}/{}页 房号列表获取为空'.format(response.request.url, item_bd['districtName'],
                                                                     item_bd['estateName'], item_bd['preSalePermit'],
                                                                     item_bd['blockName'], current_page, total_page))
        # 翻页
        if int(current_page) < int(total_page):
            next_page_temp_url = str(response.request.url).split('&page')[0] + '&page={}&roomid='
            yield scrapy.Request(
                next_page_temp_url.format(int(current_page) + 1),
                callback=self.parse_rm_li,
                meta=dict(item_bd=deepcopy(item_bd)),
                priority=6,
            )

    def parse_room_detail(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        floor_li = response.xpath("//strong[contains(text(),'所在层数')]/../text()").extract()
        floor_str = ''.join([i.strip() for i in floor_li if i and i.strip()])
        roomFloor_li = re.findall(r'第(.*)层', floor_str)
        item_rm['roomFloor'] = roomFloor_li[0] if len(roomFloor_li) else None
        floorTotalNo_li = re.findall(r'共(.*)层', floor_str)
        item_rm['floorTotalNo'] = floorTotalNo_li[0] if len(floorTotalNo_li) else None
        item_rm['roomLocation'] = response.xpath("//strong[contains(text(),'房屋坐落')]/../text()").extract_first()
        item_rm['roomUse'] = response.xpath("//strong[contains(text(),'房屋用途')]/../text()").extract_first()
        yield item_rm
